import requests,re,time
from lxml import etree
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
from wordcloud import WordCloud, ImageColorGenerator

class DoubanSpider:
    def __init__(self):
        self.ls = []

    def request(self,url,headers):
        self.ls = []
        response = requests.get(url,headers=headers)
        text = response.text
        self.get_info(text)

    def get_info(self,text):
        xml = etree.HTML(text)
        title = xml.xpath('//div[@class="hd"]/a/span[1]/text()')
        eng_title = [re.sub(r'.*/', '', i[3:]) for i in xml.xpath('//div[@class="hd"]/a/span[2]/text()')]
        href = xml.xpath('//div[@class="pic"]/a/@href')
        img_url = xml.xpath('//div[@class="pic"]/a/img/@src')
        year = re.findall(r'<p class="">.*?([0-9]{4})', text, re.S)
        score = xml.xpath('//span[@class="rating_num"]/text()')
        director = re.findall(r'<p class="">.*?导演: (.*?)[A-Z|&nbsp;]', text, re.S)
        country = re.findall(r'<p class="">.*?[0-9]{4}&nbsp;/&nbsp;(.*?)&nbsp;/', text, re.S)
        classify = re.findall(r'<p class="">.*?[0-9]{4}&nbsp;/&nbsp;.*?&nbsp;/&nbsp;([\u4e00-\u9fa5]*)\s', text, re.S)
        comment = [int(i[:-3]) for i in xml.xpath('//div[@class="bd"]/div/span[last()]/text()')]
        basic = xml.xpath('//div[@class="bd"]/p[@class="quote"]/span/text()')
        actor = [i[:-1] for i in re.findall(r'导演:.*?主演: (.*?)[A-Z|&nbsp;]', text, re.S)]
        for tit,eng_tit,h,img,y,s,d,coun,clas,c,b,a in zip(title,eng_title,href,img_url,year,score,director,country,classify,comment,basic,actor):
            self.save(tit,eng_tit,h,img,y,s,d,coun,clas,c,b,a)

    def save(self,tit,eng_tit,h,img,y,s,d,coun,clas,c,b,a):
        dic = {
            '详情链接':h,
            '图片链接':img,
            '影片中文名':tit,
            '影片外文名':eng_tit,
            '评分':s,
            '评价数':c,
            '概况':b,
            '导演':d,
            '主演':a,
            '年份': y,
            '地区':coun,
            '类别':clas
        }
        self.ls.append(dic)

if __name__ == '__main__':
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'
    }
    url0 = 'https://movie.douban.com/top250'
    spider = DoubanSpider()
    li = []
    for page in range(7):
        url =url0 + f'?start={page*25}&filter='
        spider.request(url,headers)
        ls = spider.ls
        li += ls
        print(f'已经爬完第{page+1}页')
        time.sleep(0.2)
    pd.DataFrame(li[:150]).to_excel('豆瓣排行前150电影信息.xlsx',index=False)

    font = {'family': 'SimHei'}
    matplotlib.rc('font', **font)
    df = pd.read_excel('豆瓣排行前150电影信息.xlsx')
    # 评分前10电影柱状图
    df_sort = df.sort_values(by='评分',ascending=False).head(10)
    plt.figure(figsize=(11,6))
    plt.bar(df_sort['影片中文名'],df_sort['评分'])
    plt.ylim(9.2,9.8)
    plt.xlabel('电影名称')
    plt.ylabel('评分')
    plt.savefig('前10柱状图.jpg')
    plt.show()

    # 评论数第一的电影评论词云
    img = plt.imread('背景.jpg')
    detail_url0 = list(df[df['评价数']==df['评价数'].max()]['详情链接'])[0]
    comment = ''
    for i in range(1, 11):
        detail_url = f'{detail_url0}comments?start={20*i}&limit=20&sort=new_score&status=P'
        res = requests.get(detail_url,headers=headers)
        xml = etree.HTML(res.text)
        comment += ' '.join(xml.xpath('//span[@class="short"]/text()'))
    wordcloud_name = WordCloud(
        background_color="white",
        mask=img,
        width=1000,
        height=860,
        max_words=1000,
        font_path='C:/Windows/Fonts/simkai.ttf'
    ).generate(comment)
    colors = ImageColorGenerator(np.array(img))
    wordcloud_name.recolor(color_func=colors)
    plt.figure(figsize=(10, 10))
    plt.imshow(wordcloud_name)
    plt.axis("off")
    plt.savefig('评论数第一电影评论词云.jpg')